{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "df = pd.read_csv(\"data/train_pp.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "important_num_cols = list(df.corr()[\"SalePrice\"][(\n",
    "    df.corr()[\"SalePrice\"] > 0.50) | (df.corr()[\"SalePrice\"] < -0.50)].index)\n",
    "cat_cols = [\"MSZoning\", \"Utilities\", \"BldgType\",\n",
    "            \"Heating\", \"KitchenQual\", \"SaleCondition\", \"LandSlope\"]\n",
    "important_cols = important_num_cols + cat_cols\n",
    "\n",
    "df = df[important_cols]\n",
    "X = df.drop(\"SalePrice\", axis=1)\n",
    "y = df[\"SalePrice\"]\n",
    "# One-Hot Encoding\n",
    "X = pd.get_dummies(X, columns=cat_cols)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train-Test Split\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.2, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\n",
    "# 评价函数\n",
    "\n",
    "\n",
    "def rmse_cv(model):\n",
    "    rmse = np.sqrt(-cross_val_score(model, X, y,\n",
    "                   scoring=\"neg_mean_squared_error\", cv=5)).mean()\n",
    "    return rmse\n",
    "\n",
    "\n",
    "def evaluation(y, predictions):\n",
    "    mae = mean_absolute_error(y, predictions)\n",
    "    mse = mean_squared_error(y, predictions)\n",
    "    rmse = np.sqrt(mean_squared_error(y, predictions))\n",
    "    r_squared = r2_score(y, predictions)\n",
    "    return mae, mse, rmse, r_squared\n",
    "\n",
    "\n",
    "# 标准化\n",
    "important_num_cols.remove(\"SalePrice\")\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X[important_num_cols] = scaler.fit_transform(X[important_num_cols])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support Vector Machines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型介绍"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](https://pic2.zhimg.com/v2-197913c461c1953c30b804b4a7eddfcc_1440w.jpg?source=172ae18b)\n",
    "\n",
    "在机器学习中，支持向量机（SVM，也称为支持向量网络）是具有相关学习算法的监督学习模型，用于分析数据以进行分类和回归分析。由Vladimir Vapnik与同事在AT&T 贝尔实验室开发。SVM 是最**稳健**的预测方法之一，它基于Vapnik (1982, 1995) 和 Chervonenkis (1974) 提出的统计学习框架VC理论。\n",
    "给定一组训练示例，每个训练示例都标记为属于两个类别之一，SVM 训练算法构建一个模型，将新示例分配给一个类别或另一个类别，使其成为非概率二元线性分类器。\n",
    "**SVM 将训练样本映射到空间中的点，从而最大化两个类别之间的差距宽度。然后将新示例映射到同一空间中，并根据它们落在差距的哪一侧来预测属于一个类别。**\n",
    "\n",
    "除了执行线性分类之外，SVM 还可以使用所谓的**核技巧**有效地执行非线性分类，将其输入隐式映射到高维特征空间。\n",
    "\n",
    "当数据未标记时，监督学习是不可能的，需要一种无监督学习方法，它试图找到数据到组的自然聚类，然后将新数据映射到这些形成的组。由Hava Siegelmann和Vladimir Vapnik创建的支持向量聚类算法应用支持向量机算法中开发的支持向量统计数据对未标记数据进行分类。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 优点\n",
    "\n",
    "- 在高维空间中非常高效\n",
    "- 即使在数据维度比样本数量大的情况下仍然有效\n",
    "- 在决策函数（称为支持向量）中使用训练集的子集,因此它也是高效利用内存的\n",
    "- 通用性: 不同的核函数与特定的决策函数一一对应.常见的内核已经提供,也可以指定定制的内核\n",
    "\n",
    "#### 缺点\n",
    "\n",
    "- 如果特征数量比样本数量大得多,在选择核函数时要避免过拟合,而且正则化项是非常重要的\n",
    "- 支持向量机不直接提供概率估计,这些都是使用昂贵的五次交叉验算计算的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型建立、应用与评价（sklearn）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 23429.639799121243\n",
      "MSE: 1136725417.8413253\n",
      "RMSE: 33715.35878262792\n",
      "R2 Score: 0.7895572575711571\n",
      "------------------------------\n",
      "RMSE Cross-Validation: 30756.20681426809\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVR\n",
    "svr = SVR(C=100000)\n",
    "svr.fit(X_train, y_train)\n",
    "predictions = svr.predict(X_test)\n",
    "mae, mse, rmse, r_squared = evaluation(y_test, predictions)\n",
    "print(\"MAE:\", mae)\n",
    "print(\"MSE:\", mse)\n",
    "print(\"RMSE:\", rmse)\n",
    "print(\"R2 Score:\", r_squared)\n",
    "print(\"-\"*30)\n",
    "rmse_cross_val = rmse_cv(svr)\n",
    "print(\"RMSE Cross-Validation:\", rmse_cross_val)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用PyTorch构建SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "class SVM(nn.Module):\n",
    "    \"\"\"\n",
    "    Linear Support Vector Machine\n",
    "    -----------------------------\n",
    "    This SVM is a subclass of the PyTorch nn module that\n",
    "    implements the Linear  function. The  size  of  each \n",
    "    input sample is 2 and output sample  is 1.\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        super().__init__()  # Call the init function of nn.Module\n",
    "        self.fully_connected = nn.Linear(2, 1)  # Implement the Linear function\n",
    "        \n",
    "    def forward(self, x):\n",
    "        fwd = self.fully_connected(x)  # Forward pass\n",
    "        return fwd"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "a9661a0f83c933cf25e0dd40af67627cb20747a37143c65b775945a664681eb4"
  },
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
